# -*- coding: utf-8 -*-
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exporters import JsonItemExporter
from twisted.enterprise import adbapi
import codecs
import json
import MySQLdb
import MySQLdb.cursors
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html


class ArticlespiderPipeline(object):

    def process_item(self, item, spider):
        return item


class JsonWithEncodingPipeline(object):
    # 自定义json文件的导出
    def __init__(self):
        self.file = codecs.open('article.json', 'w', encoding='utf-8')

    def process_item(self, item, spider):
        lines = json.dumps(dict(item), ensure_ascii=False)+'\n'
        self.file.write(lines)
        return item

    def spider_closed(self, spider):
        self.file.close()


class JsonExporterPipeline(object):
    # 调用scrapy提供的json export导出json文件
    def __init__(self):
        self.file = open('articleexport.json', 'wb')
        self.exporter = JsonItemExporter(self.file, encoding='utf-8', ensure_ascii=False)
        self.exporter.start_exporting()

    def close_spider(self, spider):
        self.exporter.finish_exporting()
        self.file.close()

    def process_item(self, item, spider):
        self.exporter.export_item(item=item)
        return item


class MysqlPipeline(object):
    # 采用同步的机制向mysql插入数据
    def __init__(self):
        self.conn = MySQLdb.Connect('127.0.0.1', 'root', 'Wb123456', 'articel_spider', charset='utf8', use_unicode=True)
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        insert_sql = """
                  insert into  jobbole_article(title, create_time, url, fav_nums, url_object_id)
                  VALUES (%s, %s, %s, %s, %s)
                  """
        self.cursor.execute(insert_sql, (item["title"], item["create_time"],
                                         item["url"], item["fav_nums"], item["url_object_id"]))
        self.conn.commit()


# 基于twist,提供异步操作的容器
class MysqlTwistedPipeline(object):
    def __init__(self, dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls, settings):
        dbparms = dict(
            # 下列变量名称固定写法，参见mysqldb的初始化函数注释
            host = settings["MYSQL_HOST"],
            db = settings["MYSQL_DBNAME"],
            user = settings["MYSQL_USER"],
            passwd =settings["MYSQL_PASSWORD"],
            charset='utf8',
            cursorclass = MySQLdb.cursors.DictCursor,
            use_unicode =True,
        )
        dbpool = adbapi.ConnectionPool("MySQLdb", **dbparms)
        return cls(dbpool)

    def process_item(self, item, spider):
        # 使用twist将mysql插入变成异步插入
        query = self.dbpool.runInteraction(self.do_insert, item)
        query.addErrback(self.handle_error)  # 处理异常

    def handle_error(self, failure):

        # 处理异步插入的异常
        print (failure)

    def do_insert(self, cursor, item):

        # 执行具体的插入操作
        insert_sql = """
                                 insert into  jobbole_article(title, create_time, url, fav_nums, url_object_id)
                                 VALUES (%s, %s, %s, %s, %s)
                                 """
        cursor.execute(insert_sql, (item["title"], item["create_time"], item["url"], item["fav_nums"], item["url_object_id"]))


class ArticleImagePipeline(ImagesPipeline):
    def item_completed(self, results, item, info):
        if "image_file_url" in item:
            for ok, value in results:
               image_file_path = value["path"]
            item["front_image_path"] = image_file_path
        return item
